package cc.yihy.spider.core;

import cc.yihy.spider.SpiderConfigurationProperties;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;

import javax.annotation.PostConstruct;
import java.util.List;

/**
 * @Package: cc.yihy
 * @date: 2017/11/21 14:01
 * @author: Yihy
 * @version: V1.0
 * @Description:
 */
@Component
public class JB51PageProcesser implements PageProcessor {

    @Autowired
    private SpiderConfigurationProperties properties;

    public static final String URL_POST = "http://www\\.jb51\\.net/article/[0-9]{4,10}.htm";
    public static final String URL_LIST = "http://www\\.jb51\\.net/list/list_[0-9]{1,5}_[0-9]{1,5}.htm";

    private Site site;


    @PostConstruct
    public void init() {
        site = Site.me()
                .setDomain(properties.getDomain())
                .setSleepTime(properties.getSleepTime())
                .setUserAgent(properties.getUserAgent());

    }

    @Override
    public void process(Page page) {
        List<String> links = page.getHtml().links().regex(URL_POST).all();
        page.addTargetRequests(links);


        if (page.getUrl().regex(URL_POST).match()) {

            page.putField("title", page.getHtml().xpath("//*[@id=\"contents\"]/div/div[1]/div/div[1]/h1/text()").get());
            page.putField("content", page.getHtml().$("div#content").get());

        } else {
            page.setSkip(true);
        }
        page.addTargetRequests(page.getHtml().links().regex(URL_POST).all());
        page.addTargetRequests(page.getHtml().links().regex(URL_LIST).all());

    }

    @Override
    public Site getSite() {
        return site;
    }

}
